package cn.doitedu.datayi.profile

import org.apache.spark.ml.linalg.Vectors
import org.apache.spark.ml.stat.Correlation
import org.apache.spark.sql.{DataFrame, SparkSession}

import scala.collection.mutable

/**
 * @author 涛哥
 * @nick_name "deep as the sea"
 * @contact qq:657270652 wx:doit_edu
 * @site www.doitedu.cn
 * @date 2021-01-29
 * @desc 相关性计算
 */
object CorrelationCalcDemo {

  def main(args: Array[String]): Unit = {
    val spark = SparkSession.builder()
      .master("local")
      .appName("逻辑回归")
      .getOrCreate()

    import org.apache.spark.sql.functions._
    import spark.implicits._

    val sample = spark.read.option("header", true).option("inferSchema", true).csv("user_profile/data/corr/sample")
    // sample.show(100,false)

    val arr2vec = udf((arr:mutable.WrappedArray[Double])=>{
      Vectors.dense(arr.toArray)
    })

    val vec = sample.select(arr2vec(array('material,'area,'floor,'price)) as "features")

    val cor: DataFrame = Correlation.corr(vec, "features")

    cor.show(100,false)


    spark.close()


  }


}
